#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @Time    : 2018/12/18 21:45
# @Author  : hyang
# @File    : crawlweixin.py
# @Software: python_utils

import requests
url = 'https://mp.weixin.qq.com/mp/profile_ext' \
      '?action=home' \
      '&__biz=MzA5MTAxMjEyMQ==' \
      '&scene=126&devicetype=android-23' \
      '&version=2607033c&lang=zh_CN' \
      '&nettype=WIFI' \
      '&a8scene=3' \
      '&pass_ticket=74i3DVmQKAnXdZN9T6v3ZQ4g%2FEHtxD6dafQJlz6htuXnVyg5doOi0MgMaFungz2R' \
      '&wx_header=1'

# url = 'https://mp.weixin.qq.com/mp/profile_ext' \
#       '?action=home' \
#       '&__biz=MzA5MTAxMjEyMQ==' \
#       '&scene=126&devicetype=android-23' \
#       '&version=2607033c&lang=zh_CN' \
#       '&nettype=WIFI' \
#       '&a8scene=3' \
#       '&pass_ticket=74i3DVmQKAnXdZN9T6v3ZQ4g%2FEHtxD6dafQJlz6htuXnVyg5doOi0MgMaFungz2R' \
#       '&wx_header=1'

headers ='''
Host: mp.weixin.qq.com

Connection: keep-alive

User-Agent: Mozilla/5.0 (Linux; Android 6.0.1; OPPO R9s Build/MMB29M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/57.0.2987.132 MQQBrowser/6.2 TBS/044405 Mobile Safari/537.36 MMWEBID/5576 MicroMessenger/6.7.3.1360(0x2607033C) NetType/WIFI Language/zh_CN Process/toolsmp

x-wechat-key: ac365e7126e0380a4d8698beafc8d95614cef75a1431303b6662f04b8ceb264575db354187288a3900a76b445b10b0af1c09dda1efaab48c26b3738538b3c44495c537b8dde10dccdf10d5f837155a89

x-wechat-uin: MTA1MzA1Nzk4Mw%3D%3D

Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,image/wxpic,image/sharpp,image/apng,image/tpg,/;q=0.8

Accept-Encoding: gzip, deflate

Accept-Language: zh-CN,en-US;q=0.8

Cookie: wxuin=1053057983; devicetype=android-23; version=2607033c; lang=zh_CN; pass_ticket=74i3DVmQKAnXdZN9T6v3ZQ4g/EHtxD6dafQJlz6htuXnVyg5doOi0MgMaFungz2R; wap_sid2=CL/HkfYDElx1V1pRV0lkQ2JkclkwSUNwWDNRMmx2RlUwaWpnbEE4TE9wbGxZN0d4amF3NkcxWUpyUUtfaFdkVXV0VWtVU2RxRXNhTnFUeUE1NDhZM2JpVGtvVUJxTnNEQUFBfjDZ7ePgBTgNQJVO

Q-UA2: QV=3&PL=ADR&PR=WX&PP=com.tencent.mm&PPVN=6.7.3&TBSVC=43620&CO=BK&COVC=044405&PB=GE&VE=GA&DE=PHONE&CHID=0&LCID=9422&MO= OPPOR9s &RL=1080*1920&OS=6.0.1&API=23

Q-GUID: edb298c301f35e6c59298f2313b788cb

Q-Auth: 31045b957cf33acf31e40be2f3e71c5217597676a9729f1b
'''


def headers_to_dict(headers):
    """
    将字符串
    '''
    Host: mp.weixin.qq.com
    Connection: keep-alive
    Cache-Control: max-age=
    '''
    转换成字典对象
    {
        "Host": "mp.weixin.qq.com",
        "Connection": "keep-alive",
        "Cache-Control":"max-age="
    }
    :param headers: str
    :return: dict
    """
    headers = headers.split("\n")
    d_headers = dict()
    for h in headers:
        if h:
            k, v = h.split(":", 1)
            d_headers[k] = v.strip()
    return d_headers


# with open("weixin_history.html", "w", encoding="utf-8") as f:
#     f.write(response.text)



def extract_data(html_content):
    """
    从html页面中提取历史文章数据
    :param html_content 页面源代码
    :return: 历史文章列表
    """
    import re
    import html
    import json

    rex = "msgList = '({.*?})'"
    pattern = re.compile(pattern=rex, flags=re.S)
    match = pattern.search(html_content)
    if match:
        data = match.group(1)
        data = html.unescape(data)  # 处理转义
        # print('data: {}'.format(data))
        data = json.loads(data)
        articles = data.get("list")
        # for item in articles:
        #     print(item)
        return articles


if __name__ == '__main__':
    response = requests.get(url, headers=headers_to_dict(headers), verify=False)
    print(response.text)
    if '<title>验证</title>' in response.text:
        raise Exception("获取微信公众号文章失败，可能是因为你的请求参数有误，请重新获取")
    data = extract_data(response.text)
    for item in data:
        print(item)